package com.wudl.core

import org.apache.spark
import org.apache.spark.sql.hive.test.TestHive.sparkContext
import org.apache.spark.{SparkConf, SparkContext}

/**
 *
 * @author wudl
 * @create 2021/6/8 17:57
 * @description  对几何函数进行去重
 */
object SparkDistinct {
  def main(args: Array[String]): Unit = {

    val sparkConf = new SparkConf().setMaster("local").setAppName("SparkDistinct")

    val sc  = new SparkContext(sparkConf)
    val rdd = sc.makeRDD(List(1, 2, 3, 4, 5, 2, 1))
    val result01  = rdd.distinct(2)
    val result = rdd.distinct()

    result.collect().foreach(println)
    println("****-************************")
    result01.foreach(println)



    sc.stop()



  }

}
